{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "61ea1c38-24ac-46ee-b8f5-fdeb65511131",
   "metadata": {
    "id": "61ea1c38-24ac-46ee-b8f5-fdeb65511131"
   },
   "source": [
    "# RAG Chat Bot for Hybrid Search\n",
    "\n",
    "This is the accompanying notebook for the [Oct 19 (2023) RAG for Hybrid Search meetup](https://www.pinecone.io/community/events/sf-meetup-october-2023/).\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9960f94c-11a1-4c9b-99fc-0552e3f20886",
   "metadata": {
    "id": "9960f94c-11a1-4c9b-99fc-0552e3f20886"
   },
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/pinecone-io/examples/blob/master/learn/generation/rag-for-hybrid/rag-for-hybrid-search.ipynb) [![Open nbviewer](https://raw.githubusercontent.com/pinecone-io/examples/master/assets/nbviewer-shield.svg)](https://nbviewer.org/github/pinecone-io/examples/blob/master/learn/generation/rag-for-hybrid/rag-for-hybrid-search.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67f57977-9583-4772-86d9-e28e83dbe147",
   "metadata": {
    "id": "67f57977-9583-4772-86d9-e28e83dbe147"
   },
   "source": [
    "Quick notes:\n",
    "- You will need an OpenAI API Key\n",
    "- You will need a Pinecone account (API key & environment)\n",
    "- Cells that preview data are commented out, so that users can more easily navigate the notebook on Github. Run the notebook in Colab with these cells un-commented to see data previews.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "I1Pm3RfT6hjO",
   "metadata": {
    "id": "I1Pm3RfT6hjO"
   },
   "outputs": [],
   "source": [
    "!pip3 install colab-xterm # Just makes the shell commands interactive, in case you have to press ENTER or type in 'Y/n' etc.\n",
    "%load_ext colabxterm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bc3b8d11-6a04-4e7c-ace2-2bc28ab64b7f",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bc3b8d11-6a04-4e7c-ace2-2bc28ab64b7f",
    "outputId": "a9d3e4dd-9919-4c63-8b1b-6dc7817304dc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# Install libraries\n",
    "\n",
    "%xterm\n",
    "\n",
    "!pip3 install -qU \\\n",
    "  \"pinecone-client[grpc]\"==2.2.4 \\\n",
    "  pinecone-text==0.5.4 \\\n",
    "  unstructured==0.10.24 \\\n",
    "  sentence-transformers==2.2.2 \\\n",
    "  langchain==0.0.327 \\\n",
    "  openai==0.28.1 \\\n",
    "  pdfminer.six \\\n",
    "  pdf2image==1.16.3 \\\n",
    "  python-dotenv==1.0.0 \\\n",
    "  pytesseract==0.3.10 \\\n",
    "  unstructured_pytesseract==0.3.12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5g-la06jXtsn",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5g-la06jXtsn",
    "outputId": "ac975300-dce4-4fb9-d995-2aff7463a1d8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "poppler-utils is already the newest version (22.02.0-2ubuntu0.2).\n",
      "0 upgraded, 0 newly installed, 0 to remove and 19 not upgraded.\n"
     ]
    }
   ],
   "source": [
    "!apt-get install poppler-utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6hrtPeeIa-I7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6hrtPeeIa-I7",
    "outputId": "228831e5-0623-4a2d-c247-b3c78a8a5ac6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "tesseract-ocr is already the newest version (4.1.1-2.1build1).\n",
      "0 upgraded, 0 newly installed, 0 to remove and 19 not upgraded.\n",
      "Reading package lists... Done\n",
      "Building dependency tree... Done\n",
      "Reading state information... Done\n",
      "libtesseract-dev is already the newest version (4.1.1-2.1build1).\n",
      "0 upgraded, 0 newly installed, 0 to remove and 19 not upgraded.\n"
     ]
    }
   ],
   "source": [
    "!sudo apt install tesseract-ocr\n",
    "!sudo apt install libtesseract-dev"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6af345ef-6321-4a30-b2e3-9e84cbd4d5b2",
   "metadata": {
    "id": "6af345ef-6321-4a30-b2e3-9e84cbd4d5b2"
   },
   "source": [
    "Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0520a370-63a3-4f16-8cf1-acfc8cc7f914",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0520a370-63a3-4f16-8cf1-acfc8cc7f914",
    "outputId": "a1bd3909-bf5b-41c5-b39d-2d12ce93123f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/pinecone/index.py:4: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "from pinecone import Pinecone\n",
    "import os\n",
    "import re\n",
    "from uuid import uuid4\n",
    "from typing import IO, Any, Dict, List, Tuple\n",
    "from copy import deepcopy\n",
    "import requests\n",
    "\n",
    "from unstructured.partition.pdf import partition_pdf\n",
    "from unstructured.documents.elements import Text\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain.docstore.document import Document\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from pinecone import Pinecone\n",
    "import openai\n",
    "from pinecone.core.client.model.query_response import QueryResponse\n",
    "\n",
    "from pinecone_text.sparse import BM25Encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5858c70-9d3f-437d-a289-066ae69a64e2",
   "metadata": {
    "id": "c5858c70-9d3f-437d-a289-066ae69a64e2"
   },
   "source": [
    "Set up the environment variables we'll need. We recommend using `dotenv`. It's a super simple way to keep your variables safe, but accessible. Simply create a `.env` file with your secrets in it, and use the Python `dotenv` and `os` libraries to load them.\n",
    "\n",
    "To import your `.env` file into Colab, upload it (or create it) in the `/content/` dir."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8fb61e9a-c191-489a-90e2-48df47e0c77b",
   "metadata": {
    "id": "8fb61e9a-c191-489a-90e2-48df47e0c77b"
   },
   "outputs": [],
   "source": [
    "%load_ext dotenv\n",
    "from dotenv import load_dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d62f04ff-5339-4097-b8e4-98a29450e879",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d62f04ff-5339-4097-b8e4-98a29450e879",
    "outputId": "aec7e86b-fde5-45ab-c181-296a87221ce5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make sure dotenv is in our kernel environment & working\n",
    "\n",
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e0d6d9c5-2cf2-4796-8141-564e1b94994b",
   "metadata": {
    "id": "e0d6d9c5-2cf2-4796-8141-564e1b94994b"
   },
   "outputs": [],
   "source": [
    "pinecone_api_key = os.getenv('PINECONE_API_KEY')  # You can get your Pinecone api key and env (e.g. \"us-east-1\") at app.pinecone.io\n",
    "pinecone_env = os.getenv('PINECONE_ENV')\n",
    "openai_api_key = os.getenv('OPENAI_API_KEY')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "535ecd2e-fe6e-40fb-99bf-8b8333ef01fe",
   "metadata": {
    "id": "535ecd2e-fe6e-40fb-99bf-8b8333ef01fe"
   },
   "outputs": [],
   "source": [
    "# Let's make sure our dotenv secrets loaded correctly\n",
    "\n",
    "assert len(pinecone_api_key) > 0\n",
    "assert len(pinecone_env) > 0\n",
    "assert len(openai_api_key) > 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acfaeac9-d63a-4d5a-b316-f714f032c421",
   "metadata": {
    "id": "acfaeac9-d63a-4d5a-b316-f714f032c421"
   },
   "source": [
    "# Download some articles we're interested in learning more about.\n",
    "\n",
    "Remember, hybrid search is best for knowledge that contains a lot of unique keywords that you'd like to search for, along with concepts you'd like clarity on, etc. Data that works best for this type of thing include medical data, most types of research data, data with lots of entities in it, etc.\n",
    "\n",
    "We'll be using Arxiv.org articles about different vector search algorithms for this demo. They've got lots of jargon and concepts that'll work great for hybrid search!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9cc05aea-08ac-493a-8bfe-6b0a90dbfb8f",
   "metadata": {
    "id": "9cc05aea-08ac-493a-8bfe-6b0a90dbfb8f"
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import os\n",
    "\n",
    "def get_pdf(base_url: str, filename: str):\n",
    "    \"\"\"\n",
    "    Download and write a PDF file from a github repository.\n",
    "\n",
    "    :param url: URL of Github repository containing the file you want to download & write locally.\n",
    "    \"\"\"\n",
    "    res = requests.get(base_url+filename)\n",
    "    # Check if the request was successful (HTTP status code 200)\n",
    "    if res.status_code == 200:\n",
    "      with open(filename, 'wb') as f:\n",
    "          f.write(res.content)\n",
    "          print(f\"PDF downloaded and saved as {filename}\")\n",
    "    else:\n",
    "      print(f\"Failed to download the PDF. HTTP status code: {res.status_code}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c5f32da6-5ae9-47ac-a3d3-06bcdef1dfb7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "c5f32da6-5ae9-47ac-a3d3-06bcdef1dfb7",
    "outputId": "6a9627fa-80b8-4432-ecbb-bc0c42d52450"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PDF downloaded and saved as freshdiskann_paper.pdf\n",
      "PDF downloaded and saved as hnsw_paper.pdf\n",
      "PDF downloaded and saved as ivfpq_paper.pdf\n"
     ]
    }
   ],
   "source": [
    "# Download our files to the /content/ dir in Colab\n",
    "\n",
    "github_dir = \"https://github.com/pinecone-io/examples/raw/master/learn/generation/rag-for-hybrid/\"\n",
    "filenames = [\"freshdiskann_paper.pdf\", \"hnsw_paper.pdf\", \"ivfpq_paper.pdf\"]\n",
    "\n",
    "for f in filenames:\n",
    "  get_pdf(github_dir, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "tpgyK79h8ta0",
   "metadata": {
    "id": "tpgyK79h8ta0"
   },
   "outputs": [],
   "source": [
    "# Read in our file paths\n",
    "# Note: change this path to your local dir if running this notebook locally (i.e. not on Colab)\n",
    "\n",
    "freshdisk = os.path.join(\"/content/\", filenames[0])\n",
    "hnsw = os.path.join(\"/content/\", filenames[1])\n",
    "ivfpq = os.path.join(\"/content/\", filenames[2])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8e91a7b-2287-4200-b268-da25dec4f8c9",
   "metadata": {
    "id": "c8e91a7b-2287-4200-b268-da25dec4f8c9"
   },
   "source": [
    "# Partitioning & Cleaning our PDFs\n",
    "\n",
    "This step is optional. Partitioning simply uses ML to break a document up into pages, paragraphs, the title, etc. It's a nice-to-have that allows you to exclude certain elements you might not want to index, such as an article's bibliography (although we'll keep that since it could be useful information).\n",
    "\n",
    "If you want to skip this step, you can just read the PDFs into text or json, etc. and make your chunks straight from that object(s).\n",
    "\n",
    "Note: this notebook assumes you have partitioned your PDF. If you want to run this notebook from start to finish as-is, you'll need to run this step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2fd90300-2c17-4b6d-bd12-624453c82fcc",
   "metadata": {
    "id": "2fd90300-2c17-4b6d-bd12-624453c82fcc"
   },
   "outputs": [],
   "source": [
    "# Let's partition all of our PDFs and store their partitions in a dictionary for easy retrieval & inspection later\n",
    "\n",
    "# Note: This takes a few mins to run (~12 mins; will be faster if running locally (~3 mins))\n",
    "\n",
    "partitioned_files = {\n",
    "    \"freshdisk\": partition_pdf(freshdisk, url=None, strategy = 'ocr_only'),\n",
    "    \"hnsw\": partition_pdf(hnsw, url=None, strategy = 'ocr_only'),\n",
    "    \"ivfpq\": partition_pdf(ivfpq, url=None, strategy = 'ocr_only'),\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b1cb1014-b774-42ba-8eb1-598762f3b434",
   "metadata": {
    "id": "b1cb1014-b774-42ba-8eb1-598762f3b434"
   },
   "outputs": [],
   "source": [
    "# Let's make an archived copy of partitioned_files dict so if we mess it up while cleaning, we don't have to re-ocr our PDFs:\n",
    "\n",
    "partitioned_files_copy = deepcopy(partitioned_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "ea7c9628-de78-45b1-946b-50e525e42f23",
   "metadata": {
    "id": "ea7c9628-de78-45b1-946b-50e525e42f23"
   },
   "outputs": [],
   "source": [
    "# partitioned_files.get('freshdisk')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1173112f-8449-4449-b520-2a19740fd940",
   "metadata": {
    "id": "1173112f-8449-4449-b520-2a19740fd940"
   },
   "source": [
    "You can see in the preview above that each of our PDFs now has elements classifying different parts of the text, such as `Text`, `Title`, and `EmailAddress`.\n",
    "\n",
    "Data cleaning matters a lot when it comes to hybrid search, because for the keyword-search part we care about each individual token (word).\n",
    "\n",
    "Let's filter out all of the email addresses to start with, since we don't need those for any reason."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a0236621-8c64-453a-bb8b-44b7019c6671",
   "metadata": {
    "id": "a0236621-8c64-453a-bb8b-44b7019c6671",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def remove_unwanted_categories(elements: Dict[str, List[Text]], unwanted_cat: str) -> None:\n",
    "    \"\"\"\n",
    "    Remove partitions containing an unwanted category.\n",
    "\n",
    "    :parameter elements: Partitioned pieces of our documents.\n",
    "    :parameter unwanted_cat: The name of the category we'd like filtered out.\n",
    "    \"\"\"\n",
    "    for key, value in elements.items():\n",
    "        elements[key] = [i for i in value if not i.category == unwanted_cat]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "00045ae3-4821-4200-a2fa-a3be94b216c1",
   "metadata": {
    "id": "00045ae3-4821-4200-a2fa-a3be94b216c1"
   },
   "outputs": [],
   "source": [
    "# Remove unwanted EmailAddress category from dictionary of partitioned PDFs\n",
    "\n",
    "remove_unwanted_categories(partitioned_files, 'EmailAddress')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9e35647-586a-4887-975e-bb86d091b7cd",
   "metadata": {
    "id": "f9e35647-586a-4887-975e-bb86d091b7cd"
   },
   "source": [
    "No more `EmailAddress` elements!:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "644d52c1-a0dd-4209-87c7-86f022630111",
   "metadata": {
    "id": "644d52c1-a0dd-4209-87c7-86f022630111"
   },
   "outputs": [],
   "source": [
    "# partitioned_files.get('freshdisk')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb46a39a-8dcd-46cd-bbfb-f755d1052754",
   "metadata": {
    "id": "bb46a39a-8dcd-46cd-bbfb-f755d1052754"
   },
   "source": [
    "To actually see what our elements are, we can call the `.text` attribute of each object:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbabf1e0-656e-42e2-af4a-f6a3708e1cf2",
   "metadata": {
    "id": "dbabf1e0-656e-42e2-af4a-f6a3708e1cf2"
   },
   "outputs": [],
   "source": [
    "# Text preview of what's actually in one of our dictionary items:\n",
    "\n",
    "# [i.text for i in partitioned_files.get('freshdisk')]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfade5a5-baa1-4479-8deb-8968cb8376ba",
   "metadata": {
    "id": "bfade5a5-baa1-4479-8deb-8968cb8376ba"
   },
   "source": [
    "You can see there are weird things like blank spaces, single letters, etc. as their own partitions. We don't want these either, so let's get rid of them.\n",
    "\n",
    "You can also see where some page breaks were that spanned single words -- these are identifiable by a word ending with a `- `. For these, we want to get rid of the `- ` and squish the word back together, so it makes sense.\n",
    "\n",
    "(You can also see that not all of the email addresses were caught by Unstructured's ML. It's too cumbersome to go through each doc and weed those out by hand, so we'll just have to leave them for now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "22413a4e-1b61-41c6-83f3-1c7e3cc1ff54",
   "metadata": {
    "id": "22413a4e-1b61-41c6-83f3-1c7e3cc1ff54"
   },
   "outputs": [],
   "source": [
    "# Remove empty spaces & single-letter/-digit partitions:\n",
    "\n",
    "def remove_space_and_single_partitions(elements: Dict[str, List[Text]]) -> None:\n",
    "    \"\"\"\n",
    "    Remove empty partitions & partitions with lengths of 1.\n",
    "\n",
    "    :parameter elements: Partitioned pieces of our documents.\n",
    "    \"\"\"\n",
    "    for key, value in elements.items():\n",
    "        elements[key] = [i for i in value if len(i.text.strip()) > 1 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "33e8aa65-d26e-4fc7-8f34-0ec2bc72da00",
   "metadata": {
    "id": "33e8aa65-d26e-4fc7-8f34-0ec2bc72da00"
   },
   "outputs": [],
   "source": [
    "remove_space_and_single_partitions(partitioned_files)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "562a2d57-7f94-4ff9-8ef0-96e5fa1ed7d2",
   "metadata": {
    "id": "562a2d57-7f94-4ff9-8ef0-96e5fa1ed7d2"
   },
   "source": [
    "No more single-character partitions or partitions with only whitespace, perfect!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "b6a7a897-8803-40eb-baaa-9dce221c03b3",
   "metadata": {
    "id": "b6a7a897-8803-40eb-baaa-9dce221c03b3"
   },
   "outputs": [],
   "source": [
    "# [i.text for i in partitioned_files.get('freshdisk')]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c439c74-8a6b-4962-af7e-7e8d450e5b54",
   "metadata": {
    "id": "5c439c74-8a6b-4962-af7e-7e8d450e5b54"
   },
   "source": [
    "Let's now get rid of those strange words that have been split across page breaks (e.g. `funda- mental`):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7e62b963-64eb-4b03-a50c-7ae5d98e7197",
   "metadata": {
    "id": "7e62b963-64eb-4b03-a50c-7ae5d98e7197"
   },
   "outputs": [],
   "source": [
    "# Note: this function transforms our elemenets into their text representations\n",
    "\n",
    "def rejoin_split_words(elements: Dict[str, List[Text]]) -> None:\n",
    "    \"\"\"\n",
    "    Rejoing words that are split over pagebreaks.\n",
    "\n",
    "    :parameter elements: Partitioned pieces of our documents.\n",
    "    \"\"\"\n",
    "    for key, value in elements.items():\n",
    "        elements[key] = [i.text.replace('- ', '') for i in value if '- ' in i.text]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ee954160-75f7-44b4-a5c1-5bda0037fcce",
   "metadata": {
    "id": "ee954160-75f7-44b4-a5c1-5bda0037fcce"
   },
   "outputs": [],
   "source": [
    "rejoin_split_words(partitioned_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "1eaf8942-ed2b-4053-9a51-55a678ed8909",
   "metadata": {
    "id": "1eaf8942-ed2b-4053-9a51-55a678ed8909"
   },
   "outputs": [],
   "source": [
    "# partitioned_files.get('freshdisk')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5325d394-1266-4292-bf46-1d2d41a798ad",
   "metadata": {
    "id": "5325d394-1266-4292-bf46-1d2d41a798ad"
   },
   "source": [
    "You can see now that we've sewn those split words back together:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "551b2690-d91f-44a6-9969-fe2cc2142cc5",
   "metadata": {
    "id": "551b2690-d91f-44a6-9969-fe2cc2142cc5"
   },
   "source": [
    "The last cleaning step we'll want to take is removing the inline citations, e.g. `[6, 9, 11, 16, 32, 35, 38, 43, 59]` and `[12]`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6383895a-72ec-4923-aa00-f2848822d35e",
   "metadata": {
    "id": "6383895a-72ec-4923-aa00-f2848822d35e"
   },
   "outputs": [],
   "source": [
    "def remove_inline_citation_numbers(elements: Dict[str, List[Text]]) -> None:\n",
    "    \"\"\"\n",
    "    Remove inline citation numbers from partitions.\n",
    "\n",
    "    :parameter elements: Partitioned pieces of our documents.\n",
    "    \"\"\"\n",
    "    for key, value in elements.items():\n",
    "        pattern = re.compile(r'\\[\\s*(\\d+\\s*,\\s*)*\\d+\\s*\\]')\n",
    "        elements[key] = [pattern.sub('', i) for i in value]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c59afe18-b664-4a0a-8103-91049ecb9bca",
   "metadata": {
    "id": "c59afe18-b664-4a0a-8103-91049ecb9bca"
   },
   "outputs": [],
   "source": [
    "remove_inline_citation_numbers(partitioned_files)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17d99d35-605f-4dfc-af6f-47488aa2c7ab",
   "metadata": {
    "id": "17d99d35-605f-4dfc-af6f-47488aa2c7ab"
   },
   "source": [
    "We've still got some weird numbers in there, but it's pretty good!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f6cb1c5-ea66-4be9-9246-ae8a39e4ab28",
   "metadata": {
    "id": "8f6cb1c5-ea66-4be9-9246-ae8a39e4ab28"
   },
   "outputs": [],
   "source": [
    "# partitioned_files.get('freshdisk')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a653859f-4a0e-44b8-b49d-dc904cb141c9",
   "metadata": {
    "id": "a653859f-4a0e-44b8-b49d-dc904cb141c9"
   },
   "source": [
    "Now that we've cleaned our data, we can zip all the partitions (per PDF) back together so we're starting our chunking from a single, coherent text object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7a8190b3-7d9d-45cd-8463-4e47729f9016",
   "metadata": {
    "id": "7a8190b3-7d9d-45cd-8463-4e47729f9016"
   },
   "outputs": [],
   "source": [
    "# Sew our partitions back together, per PDF:\n",
    "\n",
    "def stitch_partitions_back_together(elements: Dict[str, List[Text]]) -> None:\n",
    "    \"\"\"\n",
    "    Stitch partitions back into single string object.\n",
    "\n",
    "    :parameter elements:  Partitioned pieces of our documents.\n",
    "    \"\"\"\n",
    "    for key, value in elements.items():\n",
    "        elements[key] = ' '.join(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c7051c57-483e-4262-92a1-84dfa00c3f6e",
   "metadata": {
    "id": "c7051c57-483e-4262-92a1-84dfa00c3f6e"
   },
   "outputs": [],
   "source": [
    "stitch_partitions_back_together(partitioned_files)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afc64d40-6205-4371-9b0b-a503b8fb45ef",
   "metadata": {
    "id": "afc64d40-6205-4371-9b0b-a503b8fb45ef"
   },
   "source": [
    "Good to go! All of our PDFs are now cleaned and single globs of text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a985d1a-caa0-4e91-a510-0b8b440f1a22",
   "metadata": {
    "id": "0a985d1a-caa0-4e91-a510-0b8b440f1a22",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# partitioned_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "63780cff-5390-41a4-b53d-81772bcb800f",
   "metadata": {
    "id": "63780cff-5390-41a4-b53d-81772bcb800f"
   },
   "outputs": [],
   "source": [
    "# Let's save our cleaned files to a new variable that makes more sense w/the current state\n",
    "\n",
    "cleaned_files = partitioned_files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36bc4e5e-aa4d-42bc-9bb3-d47bb46a1a1e",
   "metadata": {
    "id": "36bc4e5e-aa4d-42bc-9bb3-d47bb46a1a1e"
   },
   "source": [
    "# Chunking our PDF content\n",
    "\n",
    "Chunking is integral to achieving great relevance with vector search, whether that's sparse vector search, dense vector search, or hybrid vector search.\n",
    "\n",
    "From our [chunking strategy post](https://www.pinecone.io/learn/chunking-strategies/):\n",
    "\n",
    "> The main reason for chunking is to ensure we’re embedding a piece of content with as little noise as possible that is still semantically relevant . . . For example, in semantic search, we index a corpus of documents, with each document containing valuable information on a specific topic. By applying an effective chunking strategy, we can ensure our search results accurately capture the essence of the user’s query. If our chunks are too small or too large, it may lead to imprecise search results or missed opportunities to surface relevant content. As a rule of thumb, if the chunk of text makes sense without the surrounding context to a human, it will make sense to the language model as well. Therefore, finding the optimal chunk size for the documents in the corpus is crucial to ensuring that the search results are accurate and relevant.\n",
    "\n",
    "We need to chunk our PDFs' (text) data into sizable chunks that are semantically coherent and dense with contextual information.\n",
    "\n",
    "We'll use LangChain's `RecusiveCharacterTextSplitter` since it's a super easy utility that makes chunking quick and customizable. You should experiment with different chunk sizes and overlap values to see how the resulting chunks differ. You want each chunk to make a reasonable amount of sense as a stand-alone data object. After some experimentation on our end, we will choose a `chunk_size` of `512` and a `chunk_overlap` of `35` (characters)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d1b8f24e-0052-41e7-9a23-8db8a6e829bb",
   "metadata": {
    "id": "d1b8f24e-0052-41e7-9a23-8db8a6e829bb"
   },
   "outputs": [],
   "source": [
    "def generate_chunks(doc: str, chunk_size: int = 512, chunk_overlap: int = 35) -> List[Document]:\n",
    "    \"\"\"\n",
    "    Generate chunks of a certain size and token overlap.\n",
    "\n",
    "    :param doc: Document we want to turn into chunks.\n",
    "    :param chunk_size: Desired size of our chunks, in tokens (words).\n",
    "    :param chunk_overlap: Desired # of tokens (words) that will overlap across chunks.\n",
    "\n",
    "    :return: Chunks representations of the given document.\n",
    "    \"\"\"\n",
    "    splitter = RecursiveCharacterTextSplitter(\n",
    "        chunk_size = chunk_size,\n",
    "        chunk_overlap = chunk_overlap\n",
    "    )\n",
    "\n",
    "    return splitter.create_documents([doc])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "667392aa-ba7b-4f3a-8fb9-2b63e6066b95",
   "metadata": {
    "id": "667392aa-ba7b-4f3a-8fb9-2b63e6066b95"
   },
   "outputs": [],
   "source": [
    "def chunk_documents(docs: Dict[str, List[Text]],  chunk_size: int = 512, chunk_overlap: int = 35) -> None:\n",
    "    \"\"\"\n",
    "    Iterate over documents and chunk each one.\n",
    "\n",
    "    :parameter docs: The documents we want to chunk.\n",
    "    :param chunk_size: Desired size of our chunks, in tokens (words).\n",
    "    :param chunk_overlap: Desired # of tokens (words) that will overlap across chunks.\n",
    "    \"\"\"\n",
    "    for key, value in docs.items():\n",
    "        chunks = generate_chunks(value)\n",
    "        docs[key] = [c.page_content for c in chunks]  # Grab the text representation of the chunks via the `page_content` attribute\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1a912cf2-4e14-47c5-a074-bcfd983056d9",
   "metadata": {
    "id": "1a912cf2-4e14-47c5-a074-bcfd983056d9"
   },
   "outputs": [],
   "source": [
    "chunk_documents(cleaned_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "94320c93-1993-40ee-850e-687a71fe3da4",
   "metadata": {
    "id": "94320c93-1993-40ee-850e-687a71fe3da4"
   },
   "outputs": [],
   "source": [
    "chunked_files = cleaned_files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c646c92-3c4e-401a-a6e7-0481501ada96",
   "metadata": {
    "id": "5c646c92-3c4e-401a-a6e7-0481501ada96"
   },
   "source": [
    "Check out our chunks!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "8ceb3744-e564-472d-a800-e50ee355681a",
   "metadata": {
    "id": "8ceb3744-e564-472d-a800-e50ee355681a"
   },
   "outputs": [],
   "source": [
    "# chunked_files\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e710a40-19b0-4957-89a6-30eaba4085d0",
   "metadata": {
    "id": "3e710a40-19b0-4957-89a6-30eaba4085d0"
   },
   "source": [
    "# Create Dense Embeddings of our Chunks\n",
    "\n",
    "Hybrid search needs both dense embeddings and sparse embeddings of the same content in order to work. Let's start with dense embeddings.\n",
    "\n",
    "We'll use the `'all-MiniLM-L12-v2'` [model](https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2) hosted by HuggingFace to create our dense embeddings. It's currently high on their [MTEB (Massive Text Embedding Benchmark) Leaderboard](https://huggingface.co/spaces/mteb/leaderboard) (Reranking section), so it's a pretty safe bet. This will output dense vectors of 384 dimensions.\n",
    "\n",
    "Note: if you're playing around with this notebook, make sure to save your chunks and embeddings (both sparse and dense) in `pkl` [files](https://stackoverflow.com/questions/11218477/how-can-i-use-pickle-to-save-a-dict-or-any-other-python-object), so that you don't have to wait for the embeddings to generate again if you want to rerun any steps in this notebook."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "079ab428-d7d5-4aab-8ed3-e05408a5bfe3",
   "metadata": {
    "id": "079ab428-d7d5-4aab-8ed3-e05408a5bfe3"
   },
   "source": [
    "We'll have to create a dense embedding of each of our PDFs' chunks:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3984fa37-32e3-43f3-9b97-b0013497a88e",
   "metadata": {
    "id": "3984fa37-32e3-43f3-9b97-b0013497a88e"
   },
   "outputs": [],
   "source": [
    "def produce_embeddings(chunks: List[str]) -> List[str]:\n",
    "    \"\"\"\n",
    "    Produce dense embeddings for each chunk.\n",
    "\n",
    "    :param chunks: The chunks we want to create dense embeddings of.\n",
    "\n",
    "    :return: Dense embeddings produced by our SentenceTransformer model `all-MiniLM-L12-v2`.\n",
    "    \"\"\"\n",
    "    model = SentenceTransformer('all-MiniLM-L12-v2')\n",
    "    embeddings = []\n",
    "    for c in chunks:\n",
    "        embedding = model.encode(c)\n",
    "        embeddings.append(embedding)\n",
    "    return embeddings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "836ae697-df61-435f-b6da-093b0151e7b1",
   "metadata": {
    "id": "836ae697-df61-435f-b6da-093b0151e7b1"
   },
   "outputs": [],
   "source": [
    "freshdisk_dembeddings = produce_embeddings(chunked_files.get('freshdisk'))  # these take ~30s min to run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "7f8bb67f-2b95-48f6-bbfa-ed3bd40a1574",
   "metadata": {
    "id": "7f8bb67f-2b95-48f6-bbfa-ed3bd40a1574"
   },
   "outputs": [],
   "source": [
    "hnsw_dembeddings = produce_embeddings(chunked_files.get('hnsw'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "fc992bc7-6c15-43e9-a88f-1f70412ea90c",
   "metadata": {
    "id": "fc992bc7-6c15-43e9-a88f-1f70412ea90c"
   },
   "outputs": [],
   "source": [
    "ivfpq_dembeddings = produce_embeddings(chunked_files.get('ivfpq'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1fe90755-9203-45e1-a47b-51bedff337e1",
   "metadata": {
    "id": "1fe90755-9203-45e1-a47b-51bedff337e1"
   },
   "outputs": [],
   "source": [
    "# We can confirm the shape of each our dense embeddings is 384:\n",
    "\n",
    "# Make binary lists to keep track of any shapes that are *not* 384\n",
    "freshdisk_assertion = [0 for i in freshdisk_dembeddings if i.shape == 384]\n",
    "hnsw_assertion = [0 for i in hnsw_dembeddings if i.shape == 384]\n",
    "ivfpq_assertion = [0 for i in ivfpq_dembeddings if i.shape == 384]\n",
    "\n",
    "# Sum up our lists. If there are any embeddings that are not of shape 384, these sums will be > 0\n",
    "assert sum(freshdisk_assertion) == 0\n",
    "assert sum(hnsw_assertion) == 0\n",
    "assert sum(ivfpq_assertion) == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cffb7427-6db0-42c4-ba3e-b589edadd3e2",
   "metadata": {
    "id": "cffb7427-6db0-42c4-ba3e-b589edadd3e2"
   },
   "source": [
    "# Create Sparse Embeddings of our Chunks\n",
    "\n",
    "Now we can create our sparse embeddings. We will use the BM25 algorithm to create our sparse embeddings. The resulting vector will represent an inverted index of the tokens in our chunks, constrained by things like chunk length.\n",
    "\n",
    "Pinecone has an awesome [text library](https://github.com/pinecone-io/pinecone-text) that makes generating these vectors super easy. We also have [a great notebook](https://colab.research.google.com/github/pinecone-io/examples/blob/master/learn/search/semantic-search/sparse/bm25/bm25-vector-generation.ipynb) all about BM25 encodings.\n",
    "\n",
    "Since we're using a ML-implemented version of BM25, we need to \"fit\" the model to our corpus. To do this, we'll combine all 3 of our PDFs together, so that the BM25 model can compute all the token frequencies etc correctly. We'll then encode each of our documents with our \"fitted\" model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2aa80a1a-bea3-4d49-90cc-33206e604899",
   "metadata": {
    "id": "2aa80a1a-bea3-4d49-90cc-33206e604899"
   },
   "outputs": [],
   "source": [
    "# Join the content of all our PDFs together into 1 large corpus\n",
    "\n",
    "corpus = \"\"\n",
    "\n",
    "for i, v in chunked_files.items():\n",
    "    corpus += ' '.join(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "fa6c477c-df4a-4b78-b0a6-421527650000",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fa6c477c-df4a-4b78-b0a6-421527650000",
    "outputId": "6440d46b-e6b8-40fe-b01f-b29e7f1398ee"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "127531"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(corpus)  # Awesome, we've got lots o' tokens here for our BM25 model to learn :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "f124295c-9da5-4195-a57f-1a53f2feb842",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67,
     "referenced_widgets": [
      "46cb7f93a5b3418299de049a988f1ca1",
      "c0bdc65bb5ba4188a8d37bbf680c9a6b",
      "95d42387ac474e2ca20c324354e7278a",
      "ded84f36e8f448a288f9a6f66779dee1",
      "ffd326f46dd649ec90f2a3fcce8e861b",
      "2ede41088b81418b8f46393063355852",
      "869e80bfc8de4f4d8a7760e950fc8bb1",
      "1d8c818e1f544b54a4532b6e45dc558f",
      "7b3a4f088c864ca089a073e7226f9683",
      "e165b164b89a4f64868c5aef9f3d9a4c",
      "02581d3f611c4d6695fdc5e80a6a2aa5"
     ]
    },
    "id": "f124295c-9da5-4195-a57f-1a53f2feb842",
    "outputId": "63e8db06-c298-4544-ff46-016fbeae4da0"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46cb7f93a5b3418299de049a988f1ca1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/127531 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<pinecone_text.sparse.bm25_encoder.BM25Encoder at 0x7fa4356fbe50>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize BM25 and fit to our corpus\n",
    "\n",
    "bm25 = BM25Encoder()\n",
    "bm25.fit(corpus)  # takes ~30s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "73b2ca25-2ac4-4712-bd8d-77607ef9c852",
   "metadata": {
    "id": "73b2ca25-2ac4-4712-bd8d-77607ef9c852"
   },
   "outputs": [],
   "source": [
    "# Create embeddings for each chunk\n",
    "freshdisk_sembeddings = [bm25.encode_documents(i) for i in chunked_files.get('freshdisk')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e4b1c9f3-9192-4e46-afa2-6b4e93ebb6a0",
   "metadata": {
    "id": "e4b1c9f3-9192-4e46-afa2-6b4e93ebb6a0"
   },
   "outputs": [],
   "source": [
    "hnsw_sembeddings = [bm25.encode_documents(i) for i in chunked_files.get('hnsw')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "87899d0e-2bcd-4fc5-8c47-f27672b8dcd9",
   "metadata": {
    "id": "87899d0e-2bcd-4fc5-8c47-f27672b8dcd9"
   },
   "outputs": [],
   "source": [
    "ivfpq_sembeddings = [bm25.encode_documents(i) for i in chunked_files.get('ivfpq')]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f43b724-acdc-4c13-b321-5c63b4a3b907",
   "metadata": {
    "id": "5f43b724-acdc-4c13-b321-5c63b4a3b907"
   },
   "source": [
    "Let's look at the sparse embeddings for one of our PDFs.\n",
    "\n",
    "You'll see that each PDF's chunks has now transformed into a dictionary with `indices` and `values` keys."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "a794f792-b1b5-458b-8f37-fd9e132cf907",
   "metadata": {
    "id": "a794f792-b1b5-458b-8f37-fd9e132cf907"
   },
   "outputs": [],
   "source": [
    "# freshdisk_sembeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "02f495a3-37b8-4573-afca-3c8bfe6fb08f",
   "metadata": {
    "id": "02f495a3-37b8-4573-afca-3c8bfe6fb08f"
   },
   "outputs": [],
   "source": [
    "# We want the # of chunks per PDF to be equal to the # of sparse embeddings we've generated. Let's check that:\n",
    "\n",
    "assert len(freshdisk_sembeddings) == len(chunked_files.get('freshdisk'))\n",
    "assert len(hnsw_sembeddings) == len(chunked_files.get('hnsw'))\n",
    "assert len(ivfpq_sembeddings) == len(chunked_files.get('ivfpq'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ece747e9-abc4-4289-aac2-b2fef0831339",
   "metadata": {
    "id": "ece747e9-abc4-4289-aac2-b2fef0831339"
   },
   "source": [
    "# Getting Our Embeddings into Pinecone\n",
    "\n",
    "Now that we have made our sparse and dense embeddings, it's time to index them into our Pinecone index.\n",
    "\n",
    "One thing to note is that only [p1 and s1 pods support hybrid search](https://docs.pinecone.io/docs/indexes). Since we're not concerned about high throughput for a demo, we'll go with s1, which is optimized for storage over throughput.\n",
    "\n",
    "Hybrid search indexes inherently also need `\"dotproduct\"` as their similarity `metric`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "afe1431b-82be-4318-9b79-b485234bf25c",
   "metadata": {
    "id": "afe1431b-82be-4318-9b79-b485234bf25c",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pinecone.init(\n",
    "   api_key=pinecone_api_key,\n",
    "   environment=pinecone_env\n",
    ")\n",
    "\n",
    "# choose a name for our index\n",
    "index_name = \"hybrid-search-demo-oct23\"\n",
    "\n",
    "# create our index\n",
    "pinecone.create_index(\n",
    "   name = index_name,\n",
    "   dimension = 384,  # must match the dimensionality of our (dense) vectors\n",
    "   metric = \"dotproduct\",\n",
    "   pod_type = \"s1\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "8840e747-8d7e-4911-8b76-73d7b852b15d",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8840e747-8d7e-4911-8b76-73d7b852b15d",
    "outputId": "50410d03-fce6-4eb2-bf8a-812cf9341f21",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IndexDescription(name='hybrid-search-demo-oct23', metric='dotproduct', replicas=1, dimension=384.0, shards=1, pods=1, pod_type='s1.x1', status={'ready': True, 'state': 'Ready'}, metadata_config=None, source_collection='')"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's confirm everything looks good with our index\n",
    "\n",
    "pinecone.describe_index('hybrid-search-demo-oct23')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bb3e59a-2656-457f-aa88-e54fa60b2a63",
   "metadata": {
    "id": "9bb3e59a-2656-457f-aa88-e54fa60b2a63"
   },
   "source": [
    "We'll create an index object out of the index we just made. We'll make this with Pinecone's [GRPC client](https://docs.pinecone.io/docs/performance-tuning#using-the-grpc-client-to-get-higher-upsert-speeds), since it's a little faster for upserts:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "5f20b672-860d-4f9a-aa9a-6aff955542d8",
   "metadata": {
    "id": "5f20b672-860d-4f9a-aa9a-6aff955542d8"
   },
   "outputs": [],
   "source": [
    "index = pinecone.Index(\"hybrid-search-demo-oct23\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89dc6979-bd52-4080-b7cc-80689b6daf91",
   "metadata": {
    "id": "89dc6979-bd52-4080-b7cc-80689b6daf91"
   },
   "source": [
    "We'll need to make unique IDs for all of our objects, which is easy with the `uuid` library in Python:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "06205558-1994-466b-b1fd-f8b5a824e742",
   "metadata": {
    "id": "06205558-1994-466b-b1fd-f8b5a824e742"
   },
   "outputs": [],
   "source": [
    "def create_ids(chunks: str) -> List[str]:\n",
    "    \"\"\"\n",
    "    Create unique IDs for each document (chunk) in our index.\n",
    "\n",
    "    :param chunks: Chunks of our PDF file.\n",
    "\n",
    "    :return: Unique IDs for chunks.\n",
    "    \"\"\"\n",
    "    return [str(uuid4()) for _ in range(len(chunks))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "468968b1-dfc1-495c-8de1-dd8c40cf43e2",
   "metadata": {
    "id": "468968b1-dfc1-495c-8de1-dd8c40cf43e2"
   },
   "outputs": [],
   "source": [
    "freshdisk_ids = create_ids(chunked_files.get('freshdisk'))\n",
    "hnsw_ids = create_ids(chunked_files.get('hnsw'))\n",
    "ivfpq_ids = create_ids(chunked_files.get('ivfpq'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a855b9ff-7dcc-4543-933f-64815f9d8972",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "a855b9ff-7dcc-4543-933f-64815f9d8972",
    "outputId": "afc0f89e-3e77-4106-abb4-488464da2ff6"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'3d945e9c-d94c-406a-a2d8-9f42c01c101b'"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's preview one of our IDs:\n",
    "\n",
    "freshdisk_ids[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "46051402-3950-4c6c-a0ab-ac6b3a08e389",
   "metadata": {
    "id": "46051402-3950-4c6c-a0ab-ac6b3a08e389"
   },
   "outputs": [],
   "source": [
    "# Let's make sure we have the same # of IDs as there are chunks:\n",
    "\n",
    "assert len(freshdisk_ids) == len(chunked_files.get('freshdisk'))\n",
    "assert len(hnsw_ids) == len(chunked_files.get('hnsw'))\n",
    "assert len(ivfpq_ids) == len(chunked_files.get('ivfpq'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7eef3d9c-94f0-40d6-8c6c-7fef149c4152",
   "metadata": {
    "id": "7eef3d9c-94f0-40d6-8c6c-7fef149c4152"
   },
   "source": [
    "Now that we have our IDs, we can make our composite sparse-dense objects that we'll index into Pinecone. These will take 4 components:\n",
    "- Our IDs\n",
    "- Our sparse embeddings\n",
    "- Our dense embeddings\n",
    "- Our chunks\n",
    "\n",
    "We'll use the actual text content of our PDFs (stored in our chunks) as metadata. This allows the end user to see the content of what's being returned by their search instead of just the sparse/dense vectors. In order to store our chunks' textual data in digestible metadata object for Pinecone, we'll want to turn each chunk into a dict that has a `'text'` key to hold the chunk value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "9f994d0a-6167-4157-8e7f-0e450d164197",
   "metadata": {
    "id": "9f994d0a-6167-4157-8e7f-0e450d164197"
   },
   "outputs": [],
   "source": [
    "def create_metadata_objs(doc: List[str]) -> List[dict[str]]:\n",
    "    \"\"\"\n",
    "    Create objects to store as metadata alongside our sparse and dense vectors in our hybird Pinecone index.\n",
    "\n",
    "    :param doc: Chunks of a document we'd like to use while creating metadata objects.\n",
    "\n",
    "    :return: Metadata objects with a \"text\" key and a value that points to the text content of each chunk.\n",
    "    \"\"\"\n",
    "    return [{'text': d} for d in doc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "874f6265-5c21-4169-a8b9-2bd65f5b4a57",
   "metadata": {
    "id": "874f6265-5c21-4169-a8b9-2bd65f5b4a57"
   },
   "outputs": [],
   "source": [
    "freshdisk_metadata = create_metadata_objs(chunked_files.get('freshdisk'))\n",
    "hnsw_metadata = create_metadata_objs(chunked_files.get('hnsw'))\n",
    "ivfpq_metadata = create_metadata_objs(chunked_files.get('ivfpq'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f6a6a557-49cc-4dbf-9c00-f02c447c4c48",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "f6a6a557-49cc-4dbf-9c00-f02c447c4c48",
    "outputId": "d35e83ba-b9b4-4920-9c16-9eecc08a02df"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 'Approximate nearest neighbor search (ANNS) is a fundamental building block in information retrieval with graphbased indices being the current state-of-the-art  and widely used in the industry. Recent advances  in graph-based indices have made it possible to index and search billion-point datasets with high recall and millisecond-level latency on a single commodity machine with an SSD. In this paper, we present the first graph-based ANNS index that reflects corpus updates into the index in real-time without'}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Preview\n",
    "\n",
    "freshdisk_metadata[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "9e2add63-efaa-4022-bf18-1472ae8aacee",
   "metadata": {
    "id": "9e2add63-efaa-4022-bf18-1472ae8aacee"
   },
   "outputs": [],
   "source": [
    "def create_composite_objs(ids: str, sembeddings: List[Dict[str, List[Any]]], dembeddings: List[float], metadata: Dict[str, str]) -> List[Dict[str, Any]]:\n",
    "    \"\"\"\n",
    "    Create objects for indexing into Pinecone. Each object contains a document ID (which corresponds to the chunk, not the larger document),\n",
    "    the chunk's sparse embedding, the chunk's dense embedding, and the chunk's corresponding metadata object.\n",
    "\n",
    "    :param ids: Unique ID of a chunk we want to index.\n",
    "    :param sembeddings: Sparse embedding representation of a chunk we want to index.\n",
    "    :param dembeddings: Dense embedding representation of a chunk we want to index.\n",
    "    :param metadata: Metadata objects with a \"text\" key and a value that points to the text content of each chunk.\n",
    "\n",
    "    :return: Composite objects in the correct format for ingest into Pinecone.\n",
    "    \"\"\"\n",
    "    to_index = []\n",
    "\n",
    "    for i in range(len(metadata)):\n",
    "        to_index_obj = {\n",
    "                'id': ids[i],\n",
    "                'sparse_values': sembeddings[i],\n",
    "                'values': dembeddings[i],\n",
    "                'metadata': metadata[i]\n",
    "            }\n",
    "        to_index.append(to_index_obj)\n",
    "    return to_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "57f44a46-a6b8-4778-bab7-429e8733c181",
   "metadata": {
    "id": "57f44a46-a6b8-4778-bab7-429e8733c181"
   },
   "outputs": [],
   "source": [
    "freshdisk_com_objs = create_composite_objs(freshdisk_ids, freshdisk_sembeddings, freshdisk_dembeddings, freshdisk_metadata)\n",
    "hnsw_com_objs = create_composite_objs(hnsw_ids, hnsw_sembeddings, hnsw_dembeddings, hnsw_metadata)\n",
    "ivfpq_com_objs = create_composite_objs(ivfpq_ids, ivfpq_sembeddings, ivfpq_dembeddings, ivfpq_metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "938a4954-ecf5-4491-87e6-597a83956024",
   "metadata": {
    "id": "938a4954-ecf5-4491-87e6-597a83956024"
   },
   "outputs": [],
   "source": [
    "# freshdisk_com_objs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db7bc28e-5ead-4d44-a2af-287ee3efd712",
   "metadata": {
    "id": "db7bc28e-5ead-4d44-a2af-287ee3efd712"
   },
   "source": [
    "Now we can index (\"upsert\") our objects into our Pinecone index!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "6aaaaf91-2353-430b-aec6-0e843c499bd7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6aaaaf91-2353-430b-aec6-0e843c499bd7",
    "outputId": "0a9e7ab7-5c8f-4dab-9b0b-2c836b7c218a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "upserted_count: 58"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index.upsert(freshdisk_com_objs)\n",
    "index.upsert(hnsw_com_objs)\n",
    "index.upsert(ivfpq_com_objs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "29feb6c2-e6ad-47c4-834a-91ba08a72927",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "29feb6c2-e6ad-47c4-834a-91ba08a72927",
    "outputId": "50f5b394-3952-4b3e-9512-61955fd07127"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dimension': 384,\n",
       " 'index_fullness': 0.0,\n",
       " 'namespaces': {'': {'vector_count': 252}},\n",
       " 'total_vector_count': 252}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Woo we have our vectors (252) in our index!\n",
    "\n",
    "index.describe_index_stats()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f16c883-9892-49a6-99b5-846b356b3258",
   "metadata": {
    "id": "4f16c883-9892-49a6-99b5-846b356b3258"
   },
   "source": [
    "# Query Our Hybrid Docs\n",
    "\n",
    "Now that we have all of our hybrid vector objects in our Pinecone index, we can issue some queries!\n",
    "\n",
    "Since issuing a query to a vector index requires the query to be vectorized in the same way as the objects in the index are vectorized (so they can match up in vector space), for hybrid queries we'll have to vectorize the query *twice*! Once as a sparse vector and once as a dense vector. We then send both of those vectors to Pinecone to get items back."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "3802bd10-1a66-4a9b-b5c6-56a2779f30d9",
   "metadata": {
    "id": "3802bd10-1a66-4a9b-b5c6-56a2779f30d9"
   },
   "outputs": [],
   "source": [
    "query = \"What are nearest neighbors?\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0303a36-1bb0-40d5-85a3-eed31d003cab",
   "metadata": {
    "id": "c0303a36-1bb0-40d5-85a3-eed31d003cab",
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "Create sparse embedding from query\n",
    "\n",
    "Note: do *not* refit the bm25 model here. We want to keep the token frequencies etc from when we fit it to the text from our PDFs!\n",
    "\n",
    "You might be wondering how the model gets \"refit\" when the corpus changes, the answer is a little complicated, but essentially this is a special implementation of BM25 (which usually runs online) that has precomputed frequencies for English words, based off the MSMarco dataset. So, when you add new docs to the corpus, you don't have to \"refit\" the BM25 model, it just finds the word frequencies in the MSMarco dataset.\n",
    "\n",
    "More here: https://github.com/pinecone-io/pinecone-text/blob/main/pinecone_text/sparse/bm25_encoder.py#L255\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "fd44e1c5-5ccc-4e94-8247-d83f2a32f02a",
   "metadata": {
    "id": "fd44e1c5-5ccc-4e94-8247-d83f2a32f02a"
   },
   "outputs": [],
   "source": [
    "query_sembedding = bm25.encode_queries(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "4644d3a3-d5e3-4e6e-8383-ccd214bd0eaf",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4644d3a3-d5e3-4e6e-8383-ccd214bd0eaf",
    "outputId": "3888c25e-f743-4db6-c3d0-669167b17fff"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'indices': [3650065742, 1196854555], 'values': [0.5, 0.5]}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cool! We can see there are only two values in here, because BM25 automatically removed stop word like \"what\" and \"is\"\n",
    "\n",
    "query_sembedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "20e4ad1b-418b-4865-9995-45bb8a351efb",
   "metadata": {
    "id": "20e4ad1b-418b-4865-9995-45bb8a351efb",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create dense embedding\n",
    "\n",
    "query_dembedding = produce_embeddings([query])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c366ef74-e1cd-46d1-b7a2-e856acb6ce8a",
   "metadata": {
    "id": "c366ef74-e1cd-46d1-b7a2-e856acb6ce8a",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# query_dembedding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06ebc12c-a6a2-4834-b6ec-749fd61f32ae",
   "metadata": {
    "id": "06ebc12c-a6a2-4834-b6ec-749fd61f32ae"
   },
   "source": [
    "Pinecone vector search has a cool user feature where you can weight the sparse vectors higher or lower (i.e. of more or less importance) than the dense vectors. This is controlled by the `alpha` parameter. An `alpha` of 0 means you're doing a totally keyword-based search (i.e. only over sparse vectors), while an `alpha` of 1 means you're doing a totally semantic search (i.e. only over dense vectors).\n",
    "\n",
    "Let's make a function that'll let us weight our vectors by alpha.\n",
    "\n",
    "(We'll also include `k`, which is the number of docs we want to retrieve)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "6b61142a-192e-4b73-87f1-eeb68e8c4998",
   "metadata": {
    "id": "6b61142a-192e-4b73-87f1-eeb68e8c4998"
   },
   "outputs": [],
   "source": [
    "# Integrate alpha and top-k\n",
    "\n",
    "def weight_by_alpha(sparse_embedding: Dict[str, List[Any]], dense_embedding: List[float], alpha: float) -> Tuple[Dict[str, List[Any]], List[float]]:\n",
    "    \"\"\"\n",
    "    Weight the values of our sparse and dense embeddings by the parameter alpha (0-1).\n",
    "\n",
    "    :param sparse_embedding: Sparse embedding representation of one of our documents (or chunks).\n",
    "    :param dense_embedding: Dense embedding representation of one of our documents (or chunks).\n",
    "    :param alpha: Weighting parameter between 0-1 that controls the impact of sparse or dense embeddings on the retrieval and ranking\n",
    "        of returned docs (chunks) in our index.\n",
    "\n",
    "    :return: Weighted sparse and dense embeddings for one of our documents (chunks).\n",
    "    \"\"\"\n",
    "    if alpha < 0 or alpha > 1:\n",
    "        raise ValueError(\"Alpha must be between 0 and 1\")\n",
    "    hsparse = {\n",
    "        'indices': sparse_embedding['indices'],\n",
    "        'values':  [v * (1 - alpha) for v in sparse_embedding['values']]\n",
    "    }\n",
    "    hdense = [v * alpha for v in dense_embedding]\n",
    "    return hsparse, hdense"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0c12e86-5a18-41cf-bce0-699df56fbc8c",
   "metadata": {
    "id": "f0c12e86-5a18-41cf-bce0-699df56fbc8c"
   },
   "source": [
    "Now let's make a function that'll query our Pinecone index while taking into account whatever `alpha` and `k` values we want to pass:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "787c9898-b1d8-4eea-a8ec-5b24215c2a27",
   "metadata": {
    "id": "787c9898-b1d8-4eea-a8ec-5b24215c2a27"
   },
   "outputs": [],
   "source": [
    "# Note this doesn't have any genAI in it yet\n",
    "\n",
    "\n",
    "def issue_hybrid_query(sparse_embedding: Dict[str, List[Any]], dense_embedding: List[float], alpha: float, top_k: int) -> QueryResponse:\n",
    "    \"\"\"\n",
    "    Send properly formatted hybrid search query to Pinecone index and get back `k` ranked results (ranked by dot product similarity, as\n",
    "        defined when we made our index).\n",
    "\n",
    "    :param sparse_embedding: Sparse embedding representation of one of our documents (or chunks).\n",
    "    :param dense_embedding: Dense embedding representation of one of our documents (or chunks).\n",
    "    :param alpha: Weighting parameter between 0-1 that controls the impact of sparse or dense embeddings on the retrieval and ranking\n",
    "        of returned docs (chunks) in our index.\n",
    "    :param top_k: The number of documents (chunks) we want back from Pinecone.\n",
    "\n",
    "    :return: QueryResponse object from Pinecone containing top-k results.\n",
    "    \"\"\"\n",
    "    scaled_sparse, scaled_dense = weight_by_alpha(sparse_embedding, dense_embedding, alpha)\n",
    "\n",
    "    result = index.query(\n",
    "        vector=scaled_dense,\n",
    "        sparse_vector=scaled_sparse,\n",
    "        top_k=top_k,\n",
    "        include_metadata=True\n",
    "    )\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "626bffe8-cce1-47b9-84d6-c827e9f5cf07",
   "metadata": {
    "id": "626bffe8-cce1-47b9-84d6-c827e9f5cf07"
   },
   "source": [
    "Let's issue a pure semantic search:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "cea14e5f-20dc-44a1-8793-17583e8689ef",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cea14e5f-20dc-44a1-8793-17583e8689ef",
    "outputId": "8e9e9f96-13c9-426d-a7a3-f897ed1a5d96",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'matches': [{'id': 'dba4822b-2258-46af-ac3a-ed3fd0ab56b8',\n",
       "              'metadata': {'text': 'to the closest neighbors in a k-NN graph '\n",
       "                                   'serve as a simple approximation of the '\n",
       "                                   'Delaunay graph  (a graph which guranties '\n",
       "                                   'that the result of a basic greedy graph '\n",
       "                                   'traversal is always the nearest neighbor). '\n",
       "                                   'Unfortunately, Delaunay graph cannot be '\n",
       "                                   'efficiently constructed without prior '\n",
       "                                   'information about the structure of a space '\n",
       "                                   ', but its approximation by the nearest '\n",
       "                                   'neighbors can be done by using only '\n",
       "                                   'distances between the stored elements. It '\n",
       "                                   'was shown that proximity graph approaches '\n",
       "                                   'with such approximation'},\n",
       "              'score': 0.061434004,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '38916f3a-d81f-4f52-8b36-77969594cc91',\n",
       "              'metadata': {'text': 'list corresponding to g-(y). It contains '\n",
       "                                   'the vector (or image) identifier and the '\n",
       "                                   'binary code (the product quantizer’s '\n",
       "                                   'indexes). Searching the nearest '\n",
       "                                   'neighbor(s) of a query x consists of 1) '\n",
       "                                   'quantize x to its w nearest neighbors in '\n",
       "                                   'the codebook qe; with these w assignments. '\n",
       "                                   'The two steps are applied to all w '\n",
       "                                   'assignments. 4) select the K nearest '\n",
       "                                   'neighbors of x based on the estimated '\n",
       "                                   'distances. This is implemented efficiently '\n",
       "                                   'by maintaining a Maxheap structure of '\n",
       "                                   'fixed capacity, that stores the AK '\n",
       "                                   'smallest'},\n",
       "              'score': 0.05882353,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': 'e17b2561-904d-46fe-85fd-0feeaed32517',\n",
       "              'metadata': {'text': '-0.1 0 0.1 0.2 0.3 difference: estimator '\n",
       "                                   'd(x,y) In our experiments, we observe that '\n",
       "                                   'the correction returns inferior results on '\n",
       "                                   'average. Therefore, we advocate the use of '\n",
       "                                   'Equation 13 for the nearest neighbor '\n",
       "                                   'search. The corrected version is useful '\n",
       "                                   'only if we are interested in the distances '\n",
       "                                   'themselves. Approximate nearest neighbor '\n",
       "                                   'search with product quantizers is fast '\n",
       "                                   '(only m additions are required per '\n",
       "                                   'distance calculation) and reduces '\n",
       "                                   'significantly the memory requirements for '\n",
       "                                   'storing the descriptors.'},\n",
       "              'score': 0.045766592,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '5a0133a9-9453-48fa-9daa-9cccf2da7f02',\n",
       "              'metadata': {'text': 'nearest neighbors is then approximated by '\n",
       "                                   'the search of the nearest neighbors in '\n",
       "                                   'terms of Hamming distances between codes. '\n",
       "                                   'In , spectral hashing (SH) is shown to '\n",
       "                                   'outperform the binary codes generated by '\n",
       "                                   'the restricted Boltzmann machine , '\n",
       "                                   'boosting and LSH. In this paper, we '\n",
       "                                   'construct short codes using quantization. '\n",
       "                                   'The goal is to estimate distances using '\n",
       "                                   'vectorto-centroid distances, i.e., the '\n",
       "                                   'query vector is not quantized, codes are '\n",
       "                                   'assigned to the database vectors only. '\n",
       "                                   'This reduces the quantization noise'},\n",
       "              'score': 0.045766592,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '89ece947-843f-4da4-84db-352353a11955',\n",
       "              'metadata': {'text': 'method includes a re-ranking stage that '\n",
       "                                   'computes the real distances for the '\n",
       "                                   'candidate nearest neighbors. For the sake '\n",
       "                                   'of comparison with FLANN, we added a '\n",
       "                                   'verification stage to our IVFADC method: '\n",
       "                                   'IVFADC queries return a shortlist of R '\n",
       "                                   'candidate nearest neighbors using the '\n",
       "                                   'distance estimation. The vectors in the '\n",
       "                                   'shortlist are re-ordered using the real '\n",
       "                                   'distance, as done in , , and the closest '\n",
       "                                   'one is returned. Note that, in this '\n",
       "                                   'experimental setup, all the vectors are '\n",
       "                                   'stored in main memory. This requirement'},\n",
       "              'score': 0.044843048,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []}],\n",
       " 'namespace': ''}"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note, for our dense embedding (`query_dembedding`), we need to grab the 1st value [0] since Pinecone expects a Numpy array when queried:\n",
    "\n",
    "issue_hybrid_query(query_sembedding, query_dembedding[0], 0.0, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d98e72fd-4ace-42dc-8bd2-d086bd0907f4",
   "metadata": {
    "id": "d98e72fd-4ace-42dc-8bd2-d086bd0907f4",
    "scrolled": true
   },
   "source": [
    "And now a pure keyword search. You can see how many more domain-specific words are in these results:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "f6146883-735d-4755-84fc-f234a92725be",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "f6146883-735d-4755-84fc-f234a92725be",
    "outputId": "c7c08eaf-a07f-46df-9788-ab6de4069983",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'matches': [{'id': 'c768e258-08f0-4fca-8f1b-470b2731ab15',\n",
       "              'metadata': {'text': '(it can be random or supplied by a '\n",
       "                                   'separate algorithm) and iteratively '\n",
       "                                   'traverse the graph. At each step of the '\n",
       "                                   'traversal the algorithm examines the '\n",
       "                                   'distances from a query to the neighbors of '\n",
       "                                   'a current base node and then selects as '\n",
       "                                   'the next base node the adjacent node that '\n",
       "                                   'minimizes the distance, while constantly '\n",
       "                                   'keeping track of the best discovered '\n",
       "                                   'neighbors. The search is terminated when '\n",
       "                                   'some stopping condition is met (e.g. the '\n",
       "                                   'number of distance calculations). Links to '\n",
       "                                   'the closest neighbors in a k-NN'},\n",
       "              'score': 0.6283351,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '1068751e-5060-46ff-9657-55b21acc338b',\n",
       "              'metadata': {'text': 'a phenomenon known as the curse of '\n",
       "                                   'dimensionality , one aims to find the '\n",
       "                                   'approximate nearest neighbors (ANN) where '\n",
       "                                   'the goal is to retrieve k neighbors that '\n",
       "                                   'are close to being optimal. The quality of '\n",
       "                                   'an ANN algorithm is judged by the '\n",
       "                                   'trade-off it provides between accuracy and '\n",
       "                                   'the hardware resources such as compute, '\n",
       "                                   'memory and I/O consumed for the search. '\n",
       "                                   'Even though this abstraction of ANN search '\n",
       "                                   'is widely studied, it does not capture '\n",
       "                                   'many important real-world scenarios where '\n",
       "                                   'user interactions with a system'},\n",
       "              'score': 0.62667763,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '6bf046fc-daee-4f25-a58e-558b9d5b2e9e',\n",
       "              'metadata': {'text': 'only C++/python library for fast '\n",
       "                                   'approximate nearest neighbors. [n.d.]._ '\n",
       "                                   'https://github.com/nmslib/hnswlib Yongjoo '\n",
       "                                   'Park, Michael Cafarella, and Barzan '\n",
       "                                   'Mozafari. 2015. NeighborSensitive Hashing. '\n",
       "                                   'Proc. VLDB Endow. 9, 3 (Nov. 2015), '\n",
       "                                   '144-155. https: '\n",
       "                                   '//doi.org/10.14778/2850583.2850589 Suhas '\n",
       "                                   'Jayaram Subramanya, Fnu Devwvrit, Rohan '\n",
       "                                   'Kadekodi, Ravishankar Krishnawamy, and '\n",
       "                                   'Harsha Vardhan Simhadri. 2019. DiskANN: '\n",
       "                                   'Fast Accurate Billion-point Nearest '\n",
       "                                   'Neighbor Search on a Single Node. In '\n",
       "                                   'Advances in Neural Information'},\n",
       "              'score': 0.6251919,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '744e96f5-23a4-49e6-80eb-1a04fe73611f',\n",
       "              'metadata': {'text': 'Constantly growing amount of the available '\n",
       "                                   'information resources has led to high '\n",
       "                                   'demand in scalable and efficient '\n",
       "                                   'similarity search data structures. One of '\n",
       "                                   'the generally used approaches for '\n",
       "                                   'information search is the K-Nearest '\n",
       "                                   'Neighbor Search (K-NNS). The K-NNS assumes '\n",
       "                                   'you have a defined distance function '\n",
       "                                   'between the data elements and aims at '\n",
       "                                   'finding the K elements from the dataset '\n",
       "                                   'which minimize the distance to a given '\n",
       "                                   'query. Such algorithms are used in many '\n",
       "                                   'applications, such as non-parametric '\n",
       "                                   'machine'},\n",
       "              'score': 0.6185979,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '252e95e6-6a29-42e1-9553-582811111554',\n",
       "              'metadata': {'text': 'overcome this issue by performing '\n",
       "                                   'approximate nearest neighbor (ANN) search. '\n",
       "                                   'The key idea This work was partly realized '\n",
       "                                   'as part of the Quaero Programme, funded by '\n",
       "                                   'OSEO, French State agency for innovation. '\n",
       "                                   'It was originally published as a technical '\n",
       "                                   'report  in August 2009. It is also related '\n",
       "                                   'to the work  on source coding for nearest '\n",
       "                                   'neighbor search. shared by these '\n",
       "                                   'algorithms is to find the NN with high '\n",
       "                                   'probability “only”, instead of probability '\n",
       "                                   '1. Most of the effort has been devoted to '\n",
       "                                   'the Euclidean'},\n",
       "              'score': 0.6094537,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []}],\n",
       " 'namespace': ''}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "issue_hybrid_query(query_sembedding, query_dembedding[0], 1.0, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "623df78c-e1db-440a-97fb-25d97e6678f0",
   "metadata": {
    "id": "623df78c-e1db-440a-97fb-25d97e6678f0"
   },
   "source": [
    "You can see the differences above: when we issue a purely semantic search, our search results are about what the idea of \"nearest neighbors\" is; in our keyword search, the vast majority of our search results are just exact-word matches for the tokens \"nearest\" and \"neighbors\". Most of them are just citations from the HNSW article's bibliography!\n",
    "\n",
    "Can we get the best of both worlds? In an ideal world, my search results would both tell me \"about\" the concept of nearest neighbors and contain things like citations that I could read more about later.\n",
    "\n",
    "Let's see if we can get a combination of semantic and keyword search by toggling our `alpha` value:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "a19923ef-af08-41cb-976b-f93fd4e0148d",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "a19923ef-af08-41cb-976b-f93fd4e0148d",
    "outputId": "1613077e-3aa6-4c81-bed1-123e01fc2e52",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'matches': [{'id': 'dba4822b-2258-46af-ac3a-ed3fd0ab56b8',\n",
       "              'metadata': {'text': 'to the closest neighbors in a k-NN graph '\n",
       "                                   'serve as a simple approximation of the '\n",
       "                                   'Delaunay graph  (a graph which guranties '\n",
       "                                   'that the result of a basic greedy graph '\n",
       "                                   'traversal is always the nearest neighbor). '\n",
       "                                   'Unfortunately, Delaunay graph cannot be '\n",
       "                                   'efficiently constructed without prior '\n",
       "                                   'information about the structure of a space '\n",
       "                                   ', but its approximation by the nearest '\n",
       "                                   'neighbors can be done by using only '\n",
       "                                   'distances between the stored elements. It '\n",
       "                                   'was shown that proximity graph approaches '\n",
       "                                   'with such approximation'},\n",
       "              'score': 0.16148452,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '6bf046fc-daee-4f25-a58e-558b9d5b2e9e',\n",
       "              'metadata': {'text': 'only C++/python library for fast '\n",
       "                                   'approximate nearest neighbors. [n.d.]._ '\n",
       "                                   'https://github.com/nmslib/hnswlib Yongjoo '\n",
       "                                   'Park, Michael Cafarella, and Barzan '\n",
       "                                   'Mozafari. 2015. NeighborSensitive Hashing. '\n",
       "                                   'Proc. VLDB Endow. 9, 3 (Nov. 2015), '\n",
       "                                   '144-155. https: '\n",
       "                                   '//doi.org/10.14778/2850583.2850589 Suhas '\n",
       "                                   'Jayaram Subramanya, Fnu Devwvrit, Rohan '\n",
       "                                   'Kadekodi, Ravishankar Krishnawamy, and '\n",
       "                                   'Harsha Vardhan Simhadri. 2019. DiskANN: '\n",
       "                                   'Fast Accurate Billion-point Nearest '\n",
       "                                   'Neighbor Search on a Single Node. In '\n",
       "                                   'Advances in Neural Information'},\n",
       "              'score': 0.15647256,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '252e95e6-6a29-42e1-9553-582811111554',\n",
       "              'metadata': {'text': 'overcome this issue by performing '\n",
       "                                   'approximate nearest neighbor (ANN) search. '\n",
       "                                   'The key idea This work was partly realized '\n",
       "                                   'as part of the Quaero Programme, funded by '\n",
       "                                   'OSEO, French State agency for innovation. '\n",
       "                                   'It was originally published as a technical '\n",
       "                                   'report  in August 2009. It is also related '\n",
       "                                   'to the work  on source coding for nearest '\n",
       "                                   'neighbor search. shared by these '\n",
       "                                   'algorithms is to find the NN with high '\n",
       "                                   'probability “only”, instead of probability '\n",
       "                                   '1. Most of the effort has been devoted to '\n",
       "                                   'the Euclidean'},\n",
       "              'score': 0.15571739,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': 'c768e258-08f0-4fca-8f1b-470b2731ab15',\n",
       "              'metadata': {'text': '(it can be random or supplied by a '\n",
       "                                   'separate algorithm) and iteratively '\n",
       "                                   'traverse the graph. At each step of the '\n",
       "                                   'traversal the algorithm examines the '\n",
       "                                   'distances from a query to the neighbors of '\n",
       "                                   'a current base node and then selects as '\n",
       "                                   'the next base node the adjacent node that '\n",
       "                                   'minimizes the distance, while constantly '\n",
       "                                   'keeping track of the best discovered '\n",
       "                                   'neighbors. The search is terminated when '\n",
       "                                   'some stopping condition is met (e.g. the '\n",
       "                                   'number of distance calculations). Links to '\n",
       "                                   'the closest neighbors in a k-NN'},\n",
       "              'score': 0.15363905,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []},\n",
       "             {'id': '1068751e-5060-46ff-9657-55b21acc338b',\n",
       "              'metadata': {'text': 'a phenomenon known as the curse of '\n",
       "                                   'dimensionality , one aims to find the '\n",
       "                                   'approximate nearest neighbors (ANN) where '\n",
       "                                   'the goal is to retrieve k neighbors that '\n",
       "                                   'are close to being optimal. The quality of '\n",
       "                                   'an ANN algorithm is judged by the '\n",
       "                                   'trade-off it provides between accuracy and '\n",
       "                                   'the hardware resources such as compute, '\n",
       "                                   'memory and I/O consumed for the search. '\n",
       "                                   'Even though this abstraction of ANN search '\n",
       "                                   'is widely studied, it does not capture '\n",
       "                                   'many important real-world scenarios where '\n",
       "                                   'user interactions with a system'},\n",
       "              'score': 0.15300983,\n",
       "              'sparse_values': {'indices': [], 'values': []},\n",
       "              'values': []}],\n",
       " 'namespace': ''}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "issue_hybrid_query(query_sembedding, query_dembedding[0], 0.2, 5)  # closer to 1.0 = closer to pure keyword search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f0e73e1-4ae3-4237-91bc-cea84a810a72",
   "metadata": {
    "id": "6f0e73e1-4ae3-4237-91bc-cea84a810a72"
   },
   "source": [
    "Amazing! You can see that our first couple search results are not very different than our pure keyword search. But when you get further down the results list, you'll see that we get an equation we can use to calculate KNN. That's a bit more useful than #3 in our pure keyword search, which is a bibliography entry. That's likely because we have semantic search in there too -- Pinecone knows we want to know \"about\" KNN, so it fetches items with lots of domain-specific terms (keyword search), but also items that demonstrate the \"aboutness\" of KNN (semantic search).\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95fec382-1f9d-4e23-abbe-9897a00d531c",
   "metadata": {
    "id": "95fec382-1f9d-4e23-abbe-9897a00d531c"
   },
   "source": [
    "# Let's take a closer look. For science!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "8b16ea1f-e70a-4c44-af0a-95666cd212d8",
   "metadata": {
    "id": "8b16ea1f-e70a-4c44-af0a-95666cd212d8"
   },
   "outputs": [],
   "source": [
    "pure_keyword = issue_hybrid_query(query_sembedding, query_dembedding[0], 1.0, 5)\n",
    "pure_semantic = issue_hybrid_query(query_sembedding, query_dembedding[0], 0.0, 5)\n",
    "hybrid_1 = issue_hybrid_query(query_sembedding, query_dembedding[0], 0.1, 5)\n",
    "hybrid_2 = issue_hybrid_query(query_sembedding, query_dembedding[0], 0.2, 5)\n",
    "hybrid_3 = issue_hybrid_query(query_sembedding, query_dembedding[0], 0.3, 5)\n",
    "hybrid_4 = issue_hybrid_query(query_sembedding, query_dembedding[0], 0.4, 5)\n",
    "hybrid_5 = issue_hybrid_query(query_sembedding, query_dembedding[0], 0.5, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "6de66f2e-7a86-4711-837b-8dfd19785e27",
   "metadata": {
    "id": "6de66f2e-7a86-4711-837b-8dfd19785e27",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Let's turn these all into dataframes and see the different rankings\n",
    "# Feel free to skip this part (it's just an interesting side journey)\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.concat([\n",
    "    pd.DataFrame([(i['metadata']['text'], i['score'], 'keyword') for i in pure_keyword.get('matches')]),\n",
    "    pd.DataFrame([(i['metadata']['text'], i['score'], 'semantic') for i in pure_semantic.get('matches')]),\n",
    "    pd.DataFrame([(i['metadata']['text'], i['score'], 'hybrid_1') for i in hybrid_1.get('matches')]),\n",
    "    pd.DataFrame([(i['metadata']['text'], i['score'], 'hybrid_2') for i in hybrid_2.get('matches')]),\n",
    "    pd.DataFrame([(i['metadata']['text'], i['score'], 'hybrid_3') for i in hybrid_3.get('matches')]),\n",
    "    pd.DataFrame([(i['metadata']['text'], i['score'], 'hybrid_4') for i in hybrid_4.get('matches')]),\n",
    "    pd.DataFrame([(i['metadata']['text'], i['score'], 'hybrid_5') for i in hybrid_5.get('matches')]),\n",
    "]).rename(columns={0: 'document', 1: 'score', 2: 'search_type'})\n",
    "\n",
    "# Note: don't pay too much attention to the \"score\" column. This really only matters within the same type of search, for ranking docs.\n",
    "# Don't use it to compare *across* different search types (e.g. keyword search isn't inherently more relevant simply because it has higher\n",
    "# scores overall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "9912ce6a-e3bf-48f1-9e51-65c8cc3d8691",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "9912ce6a-e3bf-48f1-9e51-65c8cc3d8691",
    "outputId": "e4b1575d-c338-4e7c-d18a-a4e42d14183e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-3393f06e-1f07-43ec-9d98-58d70b96c4c4\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>document</th>\n",
       "      <th>score</th>\n",
       "      <th>search_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(it can be random or supplied by a separate al...</td>\n",
       "      <td>0.628335</td>\n",
       "      <td>keyword</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a phenomenon known as the curse of dimensional...</td>\n",
       "      <td>0.626678</td>\n",
       "      <td>keyword</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>only C++/python library for fast approximate n...</td>\n",
       "      <td>0.625192</td>\n",
       "      <td>keyword</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Constantly growing amount of the available inf...</td>\n",
       "      <td>0.618598</td>\n",
       "      <td>keyword</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>overcome this issue by performing approximate ...</td>\n",
       "      <td>0.609454</td>\n",
       "      <td>keyword</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3393f06e-1f07-43ec-9d98-58d70b96c4c4')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-3393f06e-1f07-43ec-9d98-58d70b96c4c4 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-3393f06e-1f07-43ec-9d98-58d70b96c4c4');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-aa19c81a-b542-4536-bb72-edf82a19321e\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-aa19c81a-b542-4536-bb72-edf82a19321e')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-aa19c81a-b542-4536-bb72-edf82a19321e button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "                                            document     score search_type\n",
       "0  (it can be random or supplied by a separate al...  0.628335     keyword\n",
       "1  a phenomenon known as the curse of dimensional...  0.626678     keyword\n",
       "2  only C++/python library for fast approximate n...  0.625192     keyword\n",
       "3  Constantly growing amount of the available inf...  0.618598     keyword\n",
       "4  overcome this issue by performing approximate ...  0.609454     keyword"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "eb6cb84e-21b9-4cd0-b1f1-06c077ea58c0",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "eb6cb84e-21b9-4cd0-b1f1-06c077ea58c0",
    "outputId": "d8671e48-828e-497e-8da3-4daab7601b66",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-c8e6689b-e6d9-43f2-95ba-775bf85a2074\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>document</th>\n",
       "      <th>score</th>\n",
       "      <th>search_type</th>\n",
       "      <th>document_encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(it can be random or supplied by a separate al...</td>\n",
       "      <td>0.628335</td>\n",
       "      <td>keyword</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Constantly growing amount of the available inf...</td>\n",
       "      <td>0.618598</td>\n",
       "      <td>keyword</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a phenomenon known as the curse of dimensional...</td>\n",
       "      <td>0.626678</td>\n",
       "      <td>keyword</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>only C++/python library for fast approximate n...</td>\n",
       "      <td>0.625192</td>\n",
       "      <td>keyword</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>overcome this issue by performing approximate ...</td>\n",
       "      <td>0.609454</td>\n",
       "      <td>keyword</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c8e6689b-e6d9-43f2-95ba-775bf85a2074')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-c8e6689b-e6d9-43f2-95ba-775bf85a2074 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-c8e6689b-e6d9-43f2-95ba-775bf85a2074');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-efaecc9d-6055-4312-b653-cb517b4c8c5e\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-efaecc9d-6055-4312-b653-cb517b4c8c5e')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-efaecc9d-6055-4312-b653-cb517b4c8c5e button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "                                            document     score search_type  \\\n",
       "0  (it can be random or supplied by a separate al...  0.628335     keyword   \n",
       "3  Constantly growing amount of the available inf...  0.618598     keyword   \n",
       "1  a phenomenon known as the curse of dimensional...  0.626678     keyword   \n",
       "2  only C++/python library for fast approximate n...  0.625192     keyword   \n",
       "4  overcome this issue by performing approximate ...  0.609454     keyword   \n",
       "\n",
       "   document_encoded  \n",
       "0                 0  \n",
       "3                 3  \n",
       "1                 4  \n",
       "2                 8  \n",
       "4                 9  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's give each document a label so that it's easier to see their ranking differences per search type\n",
    "\n",
    "from sklearn import preprocessing\n",
    "label_encoder = preprocessing.LabelEncoder()\n",
    "df['document_encoded'] = label_encoder.fit_transform(df['document'])\n",
    "\n",
    "df.head().sort_values(['document_encoded'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "f6945cec-025c-4425-831e-0aaeb4a83cf6",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "f6945cec-025c-4425-831e-0aaeb4a83cf6",
    "outputId": "5d132b09-88e0-4693-a96c-7dbfe5aaa645",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  search_type  document_encoded     score\n",
      "0    hybrid_1                10  0.111459\n",
      "1    hybrid_1                 5  0.104663\n",
      "2    hybrid_1                 9  0.099000\n",
      "3    hybrid_1                 6  0.098692\n",
      "4    hybrid_1                 8  0.097883\n",
      "  search_type  document_encoded     score\n",
      "0    hybrid_2                10  0.161485\n",
      "1    hybrid_2                 8  0.156473\n",
      "2    hybrid_2                 9  0.155717\n",
      "3    hybrid_2                 0  0.153639\n",
      "4    hybrid_2                 4  0.153010\n",
      "  search_type  document_encoded     score\n",
      "0    hybrid_3                 8  0.215062\n",
      "1    hybrid_3                 0  0.212976\n",
      "2    hybrid_3                 9  0.212434\n",
      "3    hybrid_3                 4  0.212218\n",
      "4    hybrid_3                10  0.211510\n",
      "  search_type  document_encoded     score\n",
      "0    hybrid_4                 8  0.273652\n",
      "1    hybrid_4                 0  0.272313\n",
      "2    hybrid_4                 4  0.271427\n",
      "3    hybrid_4                 9  0.269151\n",
      "4    hybrid_4                10  0.261535\n",
      "  search_type  document_encoded     score\n",
      "0    hybrid_5                 8  0.332242\n",
      "1    hybrid_5                 0  0.331650\n",
      "2    hybrid_5                 4  0.330635\n",
      "3    hybrid_5                 9  0.325869\n",
      "4    hybrid_5                 2  0.316816\n",
      "  search_type  document_encoded     score\n",
      "0     keyword                 0  0.628335\n",
      "1     keyword                 4  0.626678\n",
      "2     keyword                 8  0.625192\n",
      "3     keyword                 3  0.618598\n",
      "4     keyword                 9  0.609454\n",
      "  search_type  document_encoded     score\n",
      "0    semantic                10  0.061434\n",
      "1    semantic                 5  0.058824\n",
      "2    semantic                 1  0.045767\n",
      "3    semantic                 7  0.045767\n",
      "4    semantic                 6  0.044843\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-74-735a76e37e3d>:1: FutureWarning: In a future version of pandas, a length 1 tuple will be returned when iterating over a groupby with a grouper equal to a list of length 1. Don't supply a list with a single grouper to avoid this warning.\n",
      "  for i, v in df.groupby(['search_type']):\n"
     ]
    }
   ],
   "source": [
    "for i, v in df.groupby(['search_type']):\n",
    "    print(v[['search_type', 'document_encoded', 'score']])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b268e1e0-d59b-4d40-a19d-f084a8b108b9",
   "metadata": {
    "id": "b268e1e0-d59b-4d40-a19d-f084a8b108b9"
   },
   "source": [
    "Above, you can see the subtle ranking differences across each search type. For the most part, `document 8` is the top documents, except in `hybrid_1`, `hybrid_2` and `semantic`. In those two search types, `document 10` is the top document.\n",
    "\n",
    "It's up to you and your stakeholders to find the ideal `alpha` for your use case(s).\n",
    "\n",
    "Directly, for our use case, it seems anything >= `alpha=0.3` gets us similar results, so the impact of `alpha` is most discernable between `0.0-0.3`.\n",
    "\n",
    "Cool!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bb6ece2-cb19-43c0-9ab5-e0c0c09b46a2",
   "metadata": {
    "editable": true,
    "id": "5bb6ece2-cb19-43c0-9ab5-e0c0c09b46a2",
    "tags": []
   },
   "source": [
    "# Incorporating GenAI\n",
    "\n",
    "Now, hybrid search is cool enough, but what if you don't want to spend time sifting through your index's search results? What if you just want a single answer to a query?\n",
    "\n",
    "That's where GenAI comes in.\n",
    "\n",
    "We will make a retrieval augmented generation (RAG) pipeline that will make this happen.\n",
    "\n",
    "Since large language models (LLMs) do not know a ton of specific information (they are trained on the general Internet), especially if the information is from PDFs that it would have to download to have access to (like what are in our index), we need to give it this information!\n",
    "\n",
    "We do this by first sending our query to our Pinecone index and grabbing some search  results. We then attach these search results to our original query and send *both* to the LLM. That way, the LLM both knows what we want to ask it & can pull from its general knowledge store *and* has a specialized knowledge store (our Pinecone search results so that it can get us extra specific information.\n",
    "\n",
    "Let's try it out:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "9cefb103-621c-4a43-b4e6-caeec5dfccc1",
   "metadata": {
    "id": "9cefb103-621c-4a43-b4e6-caeec5dfccc1"
   },
   "outputs": [],
   "source": [
    "# Let's grab the textual metadata from our search results:\n",
    "\n",
    "hybrid_context = [i.get('metadata').get('text') for i in hybrid_3.get('matches')]\n",
    "pure_keyword_context = [i.get('metadata').get('text') for i in pure_keyword.get('matches')]\n",
    "pure_semantic_context = [i.get('metadata').get('text') for i in pure_semantic.get('matches')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "11788caa-ea17-4678-93de-871f74d3057f",
   "metadata": {
    "id": "11788caa-ea17-4678-93de-871f74d3057f"
   },
   "outputs": [],
   "source": [
    "# We are then going to combine this \"context\" with our original query in a format that our LLM likes:\n",
    "\n",
    "hybrid_augmented_query = \"\\n\\n---\\n\\n\".join(hybrid_context)+\"\\n\\n-----\\n\\n\"+query\n",
    "pure_keyword_augmented_query = \"\\n\\n---\\n\\n\".join(pure_keyword_context)+\"\\n\\n-----\\n\\n\"+query\n",
    "pure_semantic_augmented_query = \"\\n\\n---\\n\\n\".join(pure_keyword_context)+\"\\n\\n-----\\n\\n\"+query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "705cdd94-39ad-493e-b55b-12516b20ea1e",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "705cdd94-39ad-493e-b55b-12516b20ea1e",
    "outputId": "5deb46c6-e9ce-4d12-b3d1-854a32e464c5",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to the closest neighbors in a k-NN graph serve as a simple approximation of the Delaunay graph  (a graph which guranties that the result of a basic greedy graph traversal is always the nearest neighbor). Unfortunately, Delaunay graph cannot be efficiently constructed without prior information about the structure of a space , but its approximation by the nearest neighbors can be done by using only distances between the stored elements. It was shown that proximity graph approaches with such approximation\n",
      "\n",
      "---\n",
      "\n",
      "only C++/python library for fast approximate nearest neighbors. [n.d.]._ https://github.com/nmslib/hnswlib Yongjoo Park, Michael Cafarella, and Barzan Mozafari. 2015. NeighborSensitive Hashing. Proc. VLDB Endow. 9, 3 (Nov. 2015), 144-155. https: //doi.org/10.14778/2850583.2850589 Suhas Jayaram Subramanya, Fnu Devwvrit, Rohan Kadekodi, Ravishankar Krishnawamy, and Harsha Vardhan Simhadri. 2019. DiskANN: Fast Accurate Billion-point Nearest Neighbor Search on a Single Node. In Advances in Neural Information\n",
      "\n",
      "---\n",
      "\n",
      "overcome this issue by performing approximate nearest neighbor (ANN) search. The key idea This work was partly realized as part of the Quaero Programme, funded by OSEO, French State agency for innovation. It was originally published as a technical report  in August 2009. It is also related to the work  on source coding for nearest neighbor search. shared by these algorithms is to find the NN with high probability “only”, instead of probability 1. Most of the effort has been devoted to the Euclidean\n",
      "\n",
      "---\n",
      "\n",
      "(it can be random or supplied by a separate algorithm) and iteratively traverse the graph. At each step of the traversal the algorithm examines the distances from a query to the neighbors of a current base node and then selects as the next base node the adjacent node that minimizes the distance, while constantly keeping track of the best discovered neighbors. The search is terminated when some stopping condition is met (e.g. the number of distance calculations). Links to the closest neighbors in a k-NN\n",
      "\n",
      "---\n",
      "\n",
      "a phenomenon known as the curse of dimensionality , one aims to find the approximate nearest neighbors (ANN) where the goal is to retrieve k neighbors that are close to being optimal. The quality of an ANN algorithm is judged by the trade-off it provides between accuracy and the hardware resources such as compute, memory and I/O consumed for the search. Even though this abstraction of ANN search is widely studied, it does not capture many important real-world scenarios where user interactions with a system\n",
      "\n",
      "-----\n",
      "\n",
      "What are nearest neighbors?\n"
     ]
    }
   ],
   "source": [
    "print(hybrid_augmented_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "384afe12-83bf-4c10-baf0-0241ecd8132f",
   "metadata": {
    "id": "384afe12-83bf-4c10-baf0-0241ecd8132f"
   },
   "outputs": [],
   "source": [
    "# We are then going to give our LLM some instructions for how to act:\n",
    "\n",
    "primer = f\"\"\"You are Q&A bot. A highly intelligent system that answers\n",
    "user questions based on the information provided by the user above\n",
    "each question. If the information can not be found in the information\n",
    "provided by the user you truthfully say \"I don't know\".\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "cf840d48-be91-4dd3-b7bb-9d0556da0b80",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cf840d48-be91-4dd3-b7bb-9d0556da0b80",
    "outputId": "a65b5574-2b78-49cc-c06b-c9122c6e2874",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<OpenAIObject chat.completion id=chatcmpl-8GZrKZLwSJ1n5TJNfloluSBP78A9s at 0x7fa432bcf7e0> JSON: {\n",
       "  \"id\": \"chatcmpl-8GZrKZLwSJ1n5TJNfloluSBP78A9s\",\n",
       "  \"object\": \"chat.completion\",\n",
       "  \"created\": 1698961782,\n",
       "  \"model\": \"gpt-4-0613\",\n",
       "  \"choices\": [\n",
       "    {\n",
       "      \"index\": 0,\n",
       "      \"message\": {\n",
       "        \"role\": \"assistant\",\n",
       "        \"content\": \"Nearest neighbors, in the context of this information, refer to the points in a given space that are closest to a specific point. In a k-NN (k-nearest neighbors) graph, for each point, connections are formed to its k nearest neighbors according to a certain distance metric. This concept is commonly used in machine learning and data analysis for algorithms such as nearest neighbor search and classification.\"\n",
       "      },\n",
       "      \"finish_reason\": \"stop\"\n",
       "    }\n",
       "  ],\n",
       "  \"usage\": {\n",
       "    \"prompt_tokens\": 633,\n",
       "    \"completion_tokens\": 80,\n",
       "    \"total_tokens\": 713\n",
       "  }\n",
       "}"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we query our LLM with our augmented query & our primer!\n",
    "\n",
    "# Our hybrid query:\n",
    "\n",
    "openai.api_key = openai_api_key\n",
    "\n",
    "\n",
    "hybrid_res = openai.ChatCompletion.create(\n",
    "    model=\"gpt-4\",\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": primer},\n",
    "        {\"role\": \"user\", \"content\": hybrid_augmented_query}\n",
    "    ]\n",
    ")\n",
    "\n",
    "hybrid_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "43a229cf-3b8c-4d75-b863-f6091c108811",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "43a229cf-3b8c-4d75-b863-f6091c108811",
    "outputId": "0954f5b2-f3fa-4318-ca33-b8a872a58303",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<OpenAIObject chat.completion id=chatcmpl-8GZtBzUBVEzf5vviy5LNjNnBGJMAH at 0x7fa438b5f100> JSON: {\n",
       "  \"id\": \"chatcmpl-8GZtBzUBVEzf5vviy5LNjNnBGJMAH\",\n",
       "  \"object\": \"chat.completion\",\n",
       "  \"created\": 1698961897,\n",
       "  \"model\": \"gpt-4-0613\",\n",
       "  \"choices\": [\n",
       "    {\n",
       "      \"index\": 0,\n",
       "      \"message\": {\n",
       "        \"role\": \"assistant\",\n",
       "        \"content\": \"Nearest neighbors in the context of data algorithms refer to the concept used in the K-Nearest Neighbor Search (K-NNS). It is based on a defined distance function between data elements, and the aim is to find the K elements from the dataset which minimize the distance to a given query. These nearest neighbors are used in many applications, such as non-parametric machine learning algorithms.\"\n",
       "      },\n",
       "      \"finish_reason\": \"stop\"\n",
       "    }\n",
       "  ],\n",
       "  \"usage\": {\n",
       "    \"prompt_tokens\": 630,\n",
       "    \"completion_tokens\": 78,\n",
       "    \"total_tokens\": 708\n",
       "  }\n",
       "}"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Our pure_keyword query:\n",
    "\n",
    "pure_keyword_res = openai.ChatCompletion.create(\n",
    "    model=\"gpt-4\",\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": primer},\n",
    "        {\"role\": \"user\", \"content\": pure_keyword_augmented_query}\n",
    "    ]\n",
    ")\n",
    "\n",
    "pure_keyword_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "c198628f-ff63-414e-bd90-eb2fdb07df47",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "c198628f-ff63-414e-bd90-eb2fdb07df47",
    "outputId": "d778a65d-14cc-4752-e989-2d0851e79900",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<OpenAIObject chat.completion id=chatcmpl-8GZsC7ZU4ADhdIcsZqlgAg36w5P2a at 0x7fa4356f0220> JSON: {\n",
       "  \"id\": \"chatcmpl-8GZsC7ZU4ADhdIcsZqlgAg36w5P2a\",\n",
       "  \"object\": \"chat.completion\",\n",
       "  \"created\": 1698961836,\n",
       "  \"model\": \"gpt-4-0613\",\n",
       "  \"choices\": [\n",
       "    {\n",
       "      \"index\": 0,\n",
       "      \"message\": {\n",
       "        \"role\": \"assistant\",\n",
       "        \"content\": \"Nearest neighbors are a concept used in many algorithms, especially in machine learning. In the context of the K-Nearest Neighbor Search (K-NNS) mentioned in the information above, nearest neighbors are the K elements from a dataset which minimize the distance to a given query. This distance is calculated using a pre-defined distance function. When finding approximate nearest neighbors (ANN), the goal is to retrieve neighbors that are close to being optimal, which provides a balance between the accuracy of the results and the computational resources used for the search.\"\n",
       "      },\n",
       "      \"finish_reason\": \"stop\"\n",
       "    }\n",
       "  ],\n",
       "  \"usage\": {\n",
       "    \"prompt_tokens\": 630,\n",
       "    \"completion_tokens\": 107,\n",
       "    \"total_tokens\": 737\n",
       "  }\n",
       "}"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Our pure_semantic query:\n",
    "\n",
    "pure_semantic_res = openai.ChatCompletion.create(\n",
    "    model=\"gpt-4\",\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": primer},\n",
    "        {\"role\": \"user\", \"content\": pure_semantic_augmented_query}\n",
    "    ]\n",
    ")\n",
    "\n",
    "pure_semantic_res"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "466f61d4-1896-4716-a8e4-b25a4eda97fe",
   "metadata": {
    "id": "466f61d4-1896-4716-a8e4-b25a4eda97fe"
   },
   "source": [
    "You can see subtle differences across the different results above. It's up to you and your stakeholders to figure out what type of search (semantic, keyword, hybrid) offers the most relevant information for your end users"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d28051b6-219a-44bf-aee7-5894d6af600b",
   "metadata": {
    "id": "d28051b6-219a-44bf-aee7-5894d6af600b"
   },
   "source": [
    "# What if we take our our Pinecone vectors altogether??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "6837f020-3147-4014-b31b-f0b070ae9a69",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6837f020-3147-4014-b31b-f0b070ae9a69",
    "outputId": "ffbd78e1-3b06-493d-be79-9f81c3a63ad2",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<OpenAIObject chat.completion id=chatcmpl-8GZuAPTG3Gk2cpLgJPly624PXv5Zp at 0x7fa4357131f0> JSON: {\n",
       "  \"id\": \"chatcmpl-8GZuAPTG3Gk2cpLgJPly624PXv5Zp\",\n",
       "  \"object\": \"chat.completion\",\n",
       "  \"created\": 1698961958,\n",
       "  \"model\": \"gpt-4-0613\",\n",
       "  \"choices\": [\n",
       "    {\n",
       "      \"index\": 0,\n",
       "      \"message\": {\n",
       "        \"role\": \"assistant\",\n",
       "        \"content\": \"I'm sorry, but I can't provide the information you're asking for because you haven't provided any context related to 'nearest neighbors'. 'Nearest neighbors' may refer to different concepts depending on the context such as in geography, data science, etc.\"\n",
       "      },\n",
       "      \"finish_reason\": \"stop\"\n",
       "    }\n",
       "  ],\n",
       "  \"usage\": {\n",
       "    \"prompt_tokens\": 69,\n",
       "    \"completion_tokens\": 52,\n",
       "    \"total_tokens\": 121\n",
       "  }\n",
       "}"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What if we issue our original query without our Pinecone vectors as context?\n",
    "\n",
    "res = openai.ChatCompletion.create(\n",
    "    model=\"gpt-4\",\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": primer},\n",
    "        {\"role\": \"user\", \"content\": query}\n",
    "    ]\n",
    ")\n",
    "\n",
    "res"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b6943af-c430-4a5c-acd7-2a55f53a8f87",
   "metadata": {
    "id": "2b6943af-c430-4a5c-acd7-2a55f53a8f87"
   },
   "source": [
    "We can see that RAG really does have a huge impact! Without our PDFs, ChatGPT doesn't know much helpful detail at all! Nor can it give us bibliographic data for articles we might want to look up later!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc32f342-33ad-47cf-a11b-4b7981405b76",
   "metadata": {
    "id": "fc32f342-33ad-47cf-a11b-4b7981405b76",
    "scrolled": true
   },
   "source": [
    "# All finished!\n",
    "\n",
    "Check out [our documentation on hybrid search](https://docs.pinecone.io/docs/hybrid-search-and-sparse-vectors) and keep building awesome things!"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "02581d3f611c4d6695fdc5e80a6a2aa5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1d8c818e1f544b54a4532b6e45dc558f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2ede41088b81418b8f46393063355852": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "46cb7f93a5b3418299de049a988f1ca1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_c0bdc65bb5ba4188a8d37bbf680c9a6b",
       "IPY_MODEL_95d42387ac474e2ca20c324354e7278a",
       "IPY_MODEL_ded84f36e8f448a288f9a6f66779dee1"
      ],
      "layout": "IPY_MODEL_ffd326f46dd649ec90f2a3fcce8e861b"
     }
    },
    "7b3a4f088c864ca089a073e7226f9683": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "869e80bfc8de4f4d8a7760e950fc8bb1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "95d42387ac474e2ca20c324354e7278a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1d8c818e1f544b54a4532b6e45dc558f",
      "max": 127531,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7b3a4f088c864ca089a073e7226f9683",
      "value": 127531
     }
    },
    "c0bdc65bb5ba4188a8d37bbf680c9a6b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2ede41088b81418b8f46393063355852",
      "placeholder": "​",
      "style": "IPY_MODEL_869e80bfc8de4f4d8a7760e950fc8bb1",
      "value": "100%"
     }
    },
    "ded84f36e8f448a288f9a6f66779dee1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e165b164b89a4f64868c5aef9f3d9a4c",
      "placeholder": "​",
      "style": "IPY_MODEL_02581d3f611c4d6695fdc5e80a6a2aa5",
      "value": " 127531/127531 [00:16&lt;00:00, 9188.60it/s]"
     }
    },
    "e165b164b89a4f64868c5aef9f3d9a4c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ffd326f46dd649ec90f2a3fcce8e861b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
